import scrapy
from scrapy.http.response.html import HtmlResponse
from scrapy.selector.unified import SelectorList
from scrapy_dome.qsbk.qsbk.items import QsbkItem


class QsbkSpiderSpider(scrapy.Spider):
    name = 'qsbk_spider'
    allowed_domains = ['qiushibaike.com']
    start_urls = ['https://www.qiushibaike.com/text/page/1/']
    base_domain = 'https://www.qiushibaike.com'

    def parse(self, response):
        duanzidivs =  response.xpath('//div[contains(@class,"mb15")]')
        print(len(duanzidivs))
        for duanzidiv in duanzidivs:
            zuoze = duanzidiv.xpath('.//h2/text()').get().strip()
            content = duanzidiv.xpath('.//div[@class="content"]//text()').getall()
            content = ''.join(content).strip()
            item = QsbkItem(zuoze=zuoze, content=content)
            yield item
            # duanzi = {'作者':zuoze, '内容':content}
        next_url = response.xpath("//ul[@class='pagination']/li[last()]/a/@href").get()
        if not next_url:
            print('已爬取完最后一页！')
            return
        else:
            yield scrapy.Request(self.base_domain + next_url, callback=self.parse)


